{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "from collections import OrderedDict\n",
    "import re\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = OrderedDict({    \n",
    "    'model=BaseModel,network=resnet152,percent_on_k_winner=0.25': 'Resnet-Dense-Kwinners',\n",
    "    'model=SparseModel,network=resnet152,percent_on_k_winner=0.25': 'Resnet-Sparse-Kwinners',\n",
    "    'model=BaseModel,network=WideResNet,percent_on_k_winner=0.25': 'WideResnet-Dense-Kwinners',\n",
    "    'model=SparseModel,network=WideResNet,percent_on_k_winner=0.25': 'WideResnet-Sparse-Kwinners',\n",
    "    'model=BaseModel,network=resnet152,percent_on_k_winner=1': 'Resnet-Dense-ReLU',\n",
    "    'model=SparseModel,network=resnet152,percent_on_k_winner=1': 'Resnet-Sparse-ReLU',\n",
    "    'model=BaseModel,network=WideResNet,percent_on_k_winner=1': 'WideResnet-Dense-ReLU',\n",
    "    'model=SparseModel,network=WideResNet,percent_on_k_winner=1': 'WideResnet-Sparse-ReLU',\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a = '23_model=BaseModel,network=resnet152,percent_on_k_winner=0.25'\n",
    "# a = re.sub(r'^\\d+_', '', a)\n",
    "# a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Resnet-Dense-Kwinners', 'Resnet-Sparse-ReLU',\n",
       "       'WideResnet-Dense-Kwinners', 'WideResnet-Dense-ReLU',\n",
       "       'WideResnet-Sparse-Kwinners', 'Resnet-Sparse-Kwinners',\n",
       "       'Resnet-Dense-ReLU', 'WideResnet-Sparse-ReLU'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = os.path.expanduser(\"~/nta/results/resnet_cifar3/noise_results.json\")\n",
    "with open(file, 'r') as f:\n",
    "    noise_results = json.load(f)\n",
    "\n",
    "mapped_results = {}\n",
    "for k,v in noise_results.items():\n",
    "    new_k = mapping[k]\n",
    "    mapped_results[new_k] = v\n",
    "\n",
    "for k, v in mapped_results.items():\n",
    "    for noise_level, accuracies in v.items():\n",
    "        mapped_results[k][noise_level] = \"{:.2f} ± {:.2f}\".format(np.mean(accuracies)*100, np.std(accuracies)*100)\n",
    "# print(mapped_results)\n",
    "\n",
    "df = pd.DataFrame.from_dict(mapped_results)\n",
    "df = df.drop('0.2')\n",
    "df.columns    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-Kwinners</th>\n",
       "      <th>Resnet-Sparse-ReLU</th>\n",
       "      <th>WideResnet-Dense-Kwinners</th>\n",
       "      <th>WideResnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Sparse-Kwinners</th>\n",
       "      <th>Resnet-Sparse-Kwinners</th>\n",
       "      <th>Resnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Sparse-ReLU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.25 ± 0.26</td>\n",
       "      <td>93.68 ± 0.27</td>\n",
       "      <td>94.72 ± 0.07</td>\n",
       "      <td>94.85 ± 0.13</td>\n",
       "      <td>94.08 ± 0.19</td>\n",
       "      <td>91.92 ± 0.20</td>\n",
       "      <td>93.52 ± 0.70</td>\n",
       "      <td>94.28 ± 0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>88.14 ± 0.60</td>\n",
       "      <td>87.54 ± 0.58</td>\n",
       "      <td>85.61 ± 0.44</td>\n",
       "      <td>84.91 ± 0.44</td>\n",
       "      <td>85.38 ± 0.83</td>\n",
       "      <td>85.96 ± 0.67</td>\n",
       "      <td>86.62 ± 1.30</td>\n",
       "      <td>85.64 ± 0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>79.69 ± 1.48</td>\n",
       "      <td>77.45 ± 0.90</td>\n",
       "      <td>74.56 ± 1.14</td>\n",
       "      <td>73.02 ± 0.63</td>\n",
       "      <td>74.34 ± 1.60</td>\n",
       "      <td>76.95 ± 1.31</td>\n",
       "      <td>76.79 ± 1.46</td>\n",
       "      <td>74.16 ± 0.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>69.43 ± 2.31</td>\n",
       "      <td>64.35 ± 1.26</td>\n",
       "      <td>61.72 ± 1.73</td>\n",
       "      <td>60.18 ± 0.94</td>\n",
       "      <td>61.46 ± 2.52</td>\n",
       "      <td>66.60 ± 1.80</td>\n",
       "      <td>64.81 ± 1.66</td>\n",
       "      <td>60.94 ± 2.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>59.34 ± 3.07</td>\n",
       "      <td>51.79 ± 1.56</td>\n",
       "      <td>49.22 ± 1.77</td>\n",
       "      <td>48.12 ± 1.22</td>\n",
       "      <td>48.98 ± 3.22</td>\n",
       "      <td>56.34 ± 1.82</td>\n",
       "      <td>53.22 ± 1.74</td>\n",
       "      <td>48.50 ± 3.19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Resnet-Dense-Kwinners Resnet-Sparse-ReLU WideResnet-Dense-Kwinners  \\\n",
       "0              93.25 ± 0.26       93.68 ± 0.27              94.72 ± 0.07   \n",
       "0.025          88.14 ± 0.60       87.54 ± 0.58              85.61 ± 0.44   \n",
       "0.05           79.69 ± 1.48       77.45 ± 0.90              74.56 ± 1.14   \n",
       "0.075          69.43 ± 2.31       64.35 ± 1.26              61.72 ± 1.73   \n",
       "0.1            59.34 ± 3.07       51.79 ± 1.56              49.22 ± 1.77   \n",
       "\n",
       "      WideResnet-Dense-ReLU WideResnet-Sparse-Kwinners Resnet-Sparse-Kwinners  \\\n",
       "0              94.85 ± 0.13               94.08 ± 0.19           91.92 ± 0.20   \n",
       "0.025          84.91 ± 0.44               85.38 ± 0.83           85.96 ± 0.67   \n",
       "0.05           73.02 ± 0.63               74.34 ± 1.60           76.95 ± 1.31   \n",
       "0.075          60.18 ± 0.94               61.46 ± 2.52           66.60 ± 1.80   \n",
       "0.1            48.12 ± 1.22               48.98 ± 3.22           56.34 ± 1.82   \n",
       "\n",
       "      Resnet-Dense-ReLU WideResnet-Sparse-ReLU  \n",
       "0          93.52 ± 0.70           94.28 ± 0.10  \n",
       "0.025      86.62 ± 1.30           85.64 ± 0.38  \n",
       "0.05       76.79 ± 1.46           74.16 ± 0.92  \n",
       "0.075      64.81 ± 1.66           60.94 ± 2.17  \n",
       "0.1        53.22 ± 1.74           48.50 ± 3.19  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-ReLU</th>\n",
       "      <th>Resnet-Sparse-Kwinners</th>\n",
       "      <th>WideResnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Sparse-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.52 ± 0.70</td>\n",
       "      <td>91.92 ± 0.20</td>\n",
       "      <td>94.85 ± 0.13</td>\n",
       "      <td>94.08 ± 0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>86.62 ± 1.30</td>\n",
       "      <td>85.96 ± 0.67</td>\n",
       "      <td>84.91 ± 0.44</td>\n",
       "      <td>85.38 ± 0.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>76.79 ± 1.46</td>\n",
       "      <td>76.95 ± 1.31</td>\n",
       "      <td>73.02 ± 0.63</td>\n",
       "      <td>74.34 ± 1.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>64.81 ± 1.66</td>\n",
       "      <td>66.60 ± 1.80</td>\n",
       "      <td>60.18 ± 0.94</td>\n",
       "      <td>61.46 ± 2.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>53.22 ± 1.74</td>\n",
       "      <td>56.34 ± 1.82</td>\n",
       "      <td>48.12 ± 1.22</td>\n",
       "      <td>48.98 ± 3.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>43.26 ± 1.64</td>\n",
       "      <td>47.17 ± 1.86</td>\n",
       "      <td>38.23 ± 1.44</td>\n",
       "      <td>38.58 ± 3.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>35.78 ± 1.59</td>\n",
       "      <td>39.74 ± 1.84</td>\n",
       "      <td>31.00 ± 1.47</td>\n",
       "      <td>30.96 ± 3.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>29.97 ± 1.57</td>\n",
       "      <td>33.86 ± 1.73</td>\n",
       "      <td>25.61 ± 1.41</td>\n",
       "      <td>25.34 ± 3.06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Resnet-Dense-ReLU Resnet-Sparse-Kwinners WideResnet-Dense-ReLU  \\\n",
       "0          93.52 ± 0.70           91.92 ± 0.20          94.85 ± 0.13   \n",
       "0.025      86.62 ± 1.30           85.96 ± 0.67          84.91 ± 0.44   \n",
       "0.05       76.79 ± 1.46           76.95 ± 1.31          73.02 ± 0.63   \n",
       "0.075      64.81 ± 1.66           66.60 ± 1.80          60.18 ± 0.94   \n",
       "0.1        53.22 ± 1.74           56.34 ± 1.82          48.12 ± 1.22   \n",
       "0.125      43.26 ± 1.64           47.17 ± 1.86          38.23 ± 1.44   \n",
       "0.15       35.78 ± 1.59           39.74 ± 1.84          31.00 ± 1.47   \n",
       "0.175      29.97 ± 1.57           33.86 ± 1.73          25.61 ± 1.41   \n",
       "\n",
       "      WideResnet-Sparse-Kwinners  \n",
       "0                   94.08 ± 0.19  \n",
       "0.025               85.38 ± 0.83  \n",
       "0.05                74.34 ± 1.60  \n",
       "0.075               61.46 ± 2.52  \n",
       "0.1                 48.98 ± 3.22  \n",
       "0.125               38.58 ± 3.43  \n",
       "0.15                30.96 ± 3.41  \n",
       "0.175               25.34 ± 3.06  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# paper comparison table\n",
    "df[['Resnet-Dense-ReLU', 'Resnet-Sparse-Kwinners', 'WideResnet-Dense-ReLU', 'WideResnet-Sparse-Kwinners']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-ReLU</th>\n",
       "      <th>Resnet-Dense-Kwinners</th>\n",
       "      <th>WideResnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Dense-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.52 ± 0.70</td>\n",
       "      <td>93.25 ± 0.26</td>\n",
       "      <td>94.85 ± 0.13</td>\n",
       "      <td>94.72 ± 0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>86.62 ± 1.30</td>\n",
       "      <td>88.14 ± 0.60</td>\n",
       "      <td>84.91 ± 0.44</td>\n",
       "      <td>85.61 ± 0.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>76.79 ± 1.46</td>\n",
       "      <td>79.69 ± 1.48</td>\n",
       "      <td>73.02 ± 0.63</td>\n",
       "      <td>74.56 ± 1.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>64.81 ± 1.66</td>\n",
       "      <td>69.43 ± 2.31</td>\n",
       "      <td>60.18 ± 0.94</td>\n",
       "      <td>61.72 ± 1.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>53.22 ± 1.74</td>\n",
       "      <td>59.34 ± 3.07</td>\n",
       "      <td>48.12 ± 1.22</td>\n",
       "      <td>49.22 ± 1.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>43.26 ± 1.64</td>\n",
       "      <td>50.20 ± 3.33</td>\n",
       "      <td>38.23 ± 1.44</td>\n",
       "      <td>38.91 ± 1.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>35.78 ± 1.59</td>\n",
       "      <td>42.68 ± 3.29</td>\n",
       "      <td>31.00 ± 1.47</td>\n",
       "      <td>31.22 ± 1.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>29.97 ± 1.57</td>\n",
       "      <td>36.62 ± 3.09</td>\n",
       "      <td>25.61 ± 1.41</td>\n",
       "      <td>25.72 ± 1.23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Resnet-Dense-ReLU Resnet-Dense-Kwinners WideResnet-Dense-ReLU  \\\n",
       "0          93.52 ± 0.70          93.25 ± 0.26          94.85 ± 0.13   \n",
       "0.025      86.62 ± 1.30          88.14 ± 0.60          84.91 ± 0.44   \n",
       "0.05       76.79 ± 1.46          79.69 ± 1.48          73.02 ± 0.63   \n",
       "0.075      64.81 ± 1.66          69.43 ± 2.31          60.18 ± 0.94   \n",
       "0.1        53.22 ± 1.74          59.34 ± 3.07          48.12 ± 1.22   \n",
       "0.125      43.26 ± 1.64          50.20 ± 3.33          38.23 ± 1.44   \n",
       "0.15       35.78 ± 1.59          42.68 ± 3.29          31.00 ± 1.47   \n",
       "0.175      29.97 ± 1.57          36.62 ± 3.09          25.61 ± 1.41   \n",
       "\n",
       "      WideResnet-Dense-Kwinners  \n",
       "0                  94.72 ± 0.07  \n",
       "0.025              85.61 ± 0.44  \n",
       "0.05               74.56 ± 1.14  \n",
       "0.075              61.72 ± 1.73  \n",
       "0.1                49.22 ± 1.77  \n",
       "0.125              38.91 ± 1.63  \n",
       "0.15               31.22 ± 1.34  \n",
       "0.175              25.72 ± 1.23  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dense weights only\n",
    "df[['Resnet-Dense-ReLU', 'Resnet-Dense-Kwinners', 'WideResnet-Dense-ReLU', 'WideResnet-Dense-Kwinners']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Sparse-ReLU</th>\n",
       "      <th>Resnet-Sparse-Kwinners</th>\n",
       "      <th>WideResnet-Sparse-ReLU</th>\n",
       "      <th>WideResnet-Sparse-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.68 ± 0.27</td>\n",
       "      <td>91.92 ± 0.20</td>\n",
       "      <td>94.28 ± 0.10</td>\n",
       "      <td>94.08 ± 0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>87.54 ± 0.58</td>\n",
       "      <td>85.96 ± 0.67</td>\n",
       "      <td>85.64 ± 0.38</td>\n",
       "      <td>85.38 ± 0.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>77.45 ± 0.90</td>\n",
       "      <td>76.95 ± 1.31</td>\n",
       "      <td>74.16 ± 0.92</td>\n",
       "      <td>74.34 ± 1.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>64.35 ± 1.26</td>\n",
       "      <td>66.60 ± 1.80</td>\n",
       "      <td>60.94 ± 2.17</td>\n",
       "      <td>61.46 ± 2.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>51.79 ± 1.56</td>\n",
       "      <td>56.34 ± 1.82</td>\n",
       "      <td>48.50 ± 3.19</td>\n",
       "      <td>48.98 ± 3.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>41.74 ± 2.00</td>\n",
       "      <td>47.17 ± 1.86</td>\n",
       "      <td>38.43 ± 3.55</td>\n",
       "      <td>38.58 ± 3.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>34.46 ± 2.08</td>\n",
       "      <td>39.74 ± 1.84</td>\n",
       "      <td>31.20 ± 3.56</td>\n",
       "      <td>30.96 ± 3.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>29.31 ± 2.15</td>\n",
       "      <td>33.86 ± 1.73</td>\n",
       "      <td>25.84 ± 3.39</td>\n",
       "      <td>25.34 ± 3.06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Resnet-Sparse-ReLU Resnet-Sparse-Kwinners WideResnet-Sparse-ReLU  \\\n",
       "0           93.68 ± 0.27           91.92 ± 0.20           94.28 ± 0.10   \n",
       "0.025       87.54 ± 0.58           85.96 ± 0.67           85.64 ± 0.38   \n",
       "0.05        77.45 ± 0.90           76.95 ± 1.31           74.16 ± 0.92   \n",
       "0.075       64.35 ± 1.26           66.60 ± 1.80           60.94 ± 2.17   \n",
       "0.1         51.79 ± 1.56           56.34 ± 1.82           48.50 ± 3.19   \n",
       "0.125       41.74 ± 2.00           47.17 ± 1.86           38.43 ± 3.55   \n",
       "0.15        34.46 ± 2.08           39.74 ± 1.84           31.20 ± 3.56   \n",
       "0.175       29.31 ± 2.15           33.86 ± 1.73           25.84 ± 3.39   \n",
       "\n",
       "      WideResnet-Sparse-Kwinners  \n",
       "0                   94.08 ± 0.19  \n",
       "0.025               85.38 ± 0.83  \n",
       "0.05                74.34 ± 1.60  \n",
       "0.075               61.46 ± 2.52  \n",
       "0.1                 48.98 ± 3.22  \n",
       "0.125               38.58 ± 3.43  \n",
       "0.15                30.96 ± 3.41  \n",
       "0.175               25.34 ± 3.06  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sparse weights only\n",
    "df[['Resnet-Sparse-ReLU', 'Resnet-Sparse-Kwinners', 'WideResnet-Sparse-ReLU', 'WideResnet-Sparse-Kwinners']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-ReLU</th>\n",
       "      <th>Resnet-Sparse-ReLU</th>\n",
       "      <th>WideResnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Sparse-ReLU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.52 ± 0.70</td>\n",
       "      <td>93.68 ± 0.27</td>\n",
       "      <td>94.85 ± 0.13</td>\n",
       "      <td>94.28 ± 0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>86.62 ± 1.30</td>\n",
       "      <td>87.54 ± 0.58</td>\n",
       "      <td>84.91 ± 0.44</td>\n",
       "      <td>85.64 ± 0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>76.79 ± 1.46</td>\n",
       "      <td>77.45 ± 0.90</td>\n",
       "      <td>73.02 ± 0.63</td>\n",
       "      <td>74.16 ± 0.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>64.81 ± 1.66</td>\n",
       "      <td>64.35 ± 1.26</td>\n",
       "      <td>60.18 ± 0.94</td>\n",
       "      <td>60.94 ± 2.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>53.22 ± 1.74</td>\n",
       "      <td>51.79 ± 1.56</td>\n",
       "      <td>48.12 ± 1.22</td>\n",
       "      <td>48.50 ± 3.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>43.26 ± 1.64</td>\n",
       "      <td>41.74 ± 2.00</td>\n",
       "      <td>38.23 ± 1.44</td>\n",
       "      <td>38.43 ± 3.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>35.78 ± 1.59</td>\n",
       "      <td>34.46 ± 2.08</td>\n",
       "      <td>31.00 ± 1.47</td>\n",
       "      <td>31.20 ± 3.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>29.97 ± 1.57</td>\n",
       "      <td>29.31 ± 2.15</td>\n",
       "      <td>25.61 ± 1.41</td>\n",
       "      <td>25.84 ± 3.39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Resnet-Dense-ReLU Resnet-Sparse-ReLU WideResnet-Dense-ReLU  \\\n",
       "0          93.52 ± 0.70       93.68 ± 0.27          94.85 ± 0.13   \n",
       "0.025      86.62 ± 1.30       87.54 ± 0.58          84.91 ± 0.44   \n",
       "0.05       76.79 ± 1.46       77.45 ± 0.90          73.02 ± 0.63   \n",
       "0.075      64.81 ± 1.66       64.35 ± 1.26          60.18 ± 0.94   \n",
       "0.1        53.22 ± 1.74       51.79 ± 1.56          48.12 ± 1.22   \n",
       "0.125      43.26 ± 1.64       41.74 ± 2.00          38.23 ± 1.44   \n",
       "0.15       35.78 ± 1.59       34.46 ± 2.08          31.00 ± 1.47   \n",
       "0.175      29.97 ± 1.57       29.31 ± 2.15          25.61 ± 1.41   \n",
       "\n",
       "      WideResnet-Sparse-ReLU  \n",
       "0               94.28 ± 0.10  \n",
       "0.025           85.64 ± 0.38  \n",
       "0.05            74.16 ± 0.92  \n",
       "0.075           60.94 ± 2.17  \n",
       "0.1             48.50 ± 3.19  \n",
       "0.125           38.43 ± 3.55  \n",
       "0.15            31.20 ± 3.56  \n",
       "0.175           25.84 ± 3.39  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# relu only\n",
    "df[['Resnet-Dense-ReLU', 'Resnet-Sparse-ReLU', 'WideResnet-Dense-ReLU', 'WideResnet-Sparse-ReLU']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-Kwinners</th>\n",
       "      <th>Resnet-Sparse-Kwinners</th>\n",
       "      <th>WideResnet-Dense-Kwinners</th>\n",
       "      <th>WideResnet-Sparse-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.25 ± 0.26</td>\n",
       "      <td>91.92 ± 0.20</td>\n",
       "      <td>94.72 ± 0.07</td>\n",
       "      <td>94.08 ± 0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>88.14 ± 0.60</td>\n",
       "      <td>85.96 ± 0.67</td>\n",
       "      <td>85.61 ± 0.44</td>\n",
       "      <td>85.38 ± 0.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>79.69 ± 1.48</td>\n",
       "      <td>76.95 ± 1.31</td>\n",
       "      <td>74.56 ± 1.14</td>\n",
       "      <td>74.34 ± 1.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>69.43 ± 2.31</td>\n",
       "      <td>66.60 ± 1.80</td>\n",
       "      <td>61.72 ± 1.73</td>\n",
       "      <td>61.46 ± 2.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>59.34 ± 3.07</td>\n",
       "      <td>56.34 ± 1.82</td>\n",
       "      <td>49.22 ± 1.77</td>\n",
       "      <td>48.98 ± 3.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>50.20 ± 3.33</td>\n",
       "      <td>47.17 ± 1.86</td>\n",
       "      <td>38.91 ± 1.63</td>\n",
       "      <td>38.58 ± 3.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>42.68 ± 3.29</td>\n",
       "      <td>39.74 ± 1.84</td>\n",
       "      <td>31.22 ± 1.34</td>\n",
       "      <td>30.96 ± 3.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>36.62 ± 3.09</td>\n",
       "      <td>33.86 ± 1.73</td>\n",
       "      <td>25.72 ± 1.23</td>\n",
       "      <td>25.34 ± 3.06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Resnet-Dense-Kwinners Resnet-Sparse-Kwinners WideResnet-Dense-Kwinners  \\\n",
       "0              93.25 ± 0.26           91.92 ± 0.20              94.72 ± 0.07   \n",
       "0.025          88.14 ± 0.60           85.96 ± 0.67              85.61 ± 0.44   \n",
       "0.05           79.69 ± 1.48           76.95 ± 1.31              74.56 ± 1.14   \n",
       "0.075          69.43 ± 2.31           66.60 ± 1.80              61.72 ± 1.73   \n",
       "0.1            59.34 ± 3.07           56.34 ± 1.82              49.22 ± 1.77   \n",
       "0.125          50.20 ± 3.33           47.17 ± 1.86              38.91 ± 1.63   \n",
       "0.15           42.68 ± 3.29           39.74 ± 1.84              31.22 ± 1.34   \n",
       "0.175          36.62 ± 3.09           33.86 ± 1.73              25.72 ± 1.23   \n",
       "\n",
       "      WideResnet-Sparse-Kwinners  \n",
       "0                   94.08 ± 0.19  \n",
       "0.025               85.38 ± 0.83  \n",
       "0.05                74.34 ± 1.60  \n",
       "0.075               61.46 ± 2.52  \n",
       "0.1                 48.98 ± 3.22  \n",
       "0.125               38.58 ± 3.43  \n",
       "0.15                30.96 ± 3.41  \n",
       "0.175               25.34 ± 3.06  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# kwinners only\n",
    "df[['Resnet-Dense-Kwinners', 'Resnet-Sparse-Kwinners', 'WideResnet-Dense-Kwinners', 'WideResnet-Sparse-Kwinners']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
